#!/usr/bin/env python
# coding: utf-8

# importing module
import os
import json
from os import listdir
from os.path import isfile, join
import youtube_dl
import pandas as pd


class MyLogger():
    """logger to send errors from youtube-dl to the abyss"""
    def debug(self, msg):
        pass

    def warning(self, msg):
        pass

    def error(self, msg):
        pass


chan = pd.read_csv("./4chan-yt-frequencies.csv")
red = pd.read_csv("./reddit-yt-frequencies.csv")
ain = pd.read_csv("./AIN-channel-stats.tab", sep='\t', lineterminator='\n')
ydl = youtube_dl.YoutubeDL({'outtmpl': '%(id)s%(ext)s', 'logger': MyLogger()})

# make directories if don't exist
if not os.path.isdir("./redditvids/"):
    os.makedirs("./redditvids")
if not os.path.isdir("./4chanvids/"):
    os.makedirs("./4chanvids")


def get_info(video):
    """Get video info with ydl"""
    try:
        result = ydl.extract_info(video, download=False)
        return result
    except:
        pass


chan = chan[~chan['videoid'].str.contains("channel")]
red = red[~red['videoid'].str.contains("channel")]
chan = chan[chan['videoid'] != None]
red = red[red['videoid'] != None]
with ydl:
    for i in chan['videoid']:
        try:
            info = json.dumps(get_info(i))
            dic = json.loads(info)
            pairs = {k: dic[k] for k in list(dic)[:12]}
            with open("4chanvids/"+i+"-4chan.json", 'w') as file:
                json.dump(pairs, file, ensure_ascii=True, indent=4)
        except:
            continue

    for g in red['videoid']:
        try:
            info = json.dumps(get_info(g))
            dic = json.loads(info)
            pairs = {k: dic[k] for k in list(dic)[:12]}
            with open("redditvids/"+g+"-reddit.json", 'w') as file:
                json.dump(pairs, file, ensure_ascii=True, indent=4)
        except:
            continue


def load_json_to_df(folder):
    """read a path containing json files into a dataframe"""
    path = os.getcwd()
    file_list = [f for f in listdir(path+"/"+folder) if
                 isfile(join(path+"/"+folder, f))]
    temp = pd.DataFrame()
    for f in file_list:
        with open(os.getcwd()+"/"+folder+"/"+f) as jsonfile:
            jsondata = json.load(jsonfile)
            print(jsondata)
            print("\n")
            data = pd.DataFrame(jsondata, index=[0])
            temp = temp.append(data, ignore_index=True)
    temp.reset_index()
    return temp


Fdf = load_json_to_df('4chanvids')
Rdf = load_json_to_df('redditvids')
Fdf.to_csv('4chanvids_full.csv')
Rdf.to_csv('redditvids_full.csv')
